{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summarize results\n",
    "\n",
    "We provide the details of our test results (`scores/*.csv`) and how we summarized those results in this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import re\n",
    "import json\n",
    "import seaborn as sns\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "pd.options.display.float_format = '{:.3f}'.format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic results on Table II"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 60\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "    &                                      &  wacc &   uar &  r\\_Present &  r\\_Unknown &  r\\_Absent &  count \\\\\n",
      "model & ptconf &       &       &            &            &           &        \\\\\n",
      "\\midrule\n",
      "AST & pretrained\\_models & 0.654 & 0.672 &      0.744 &      0.769 &     0.505 &     15 \\\\\n",
      "BYOLA & pretrained\\_weights & 0.556 & 0.556 &      0.590 &      0.573 &     0.507 &     15 \\\\\n",
      "Cnn14 & external & 0.582 & 0.548 &      0.750 &      0.506 &     0.388 &     15 \\\\\n",
      "M2D & m2d\\_vit\\_base-80x608p16x16-220930-mr7 & 0.832 & 0.713 &      0.911 &      0.361 &     0.868 &     15 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wacc</th>\n",
       "      <th>uar</th>\n",
       "      <th>r_Present</th>\n",
       "      <th>r_Unknown</th>\n",
       "      <th>r_Absent</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>ptconf</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AST</th>\n",
       "      <th>pretrained_models</th>\n",
       "      <td>0.654</td>\n",
       "      <td>0.672</td>\n",
       "      <td>0.744</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.505</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BYOLA</th>\n",
       "      <th>pretrained_weights</th>\n",
       "      <td>0.556</td>\n",
       "      <td>0.556</td>\n",
       "      <td>0.590</td>\n",
       "      <td>0.573</td>\n",
       "      <td>0.507</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cnn14</th>\n",
       "      <th>external</th>\n",
       "      <td>0.582</td>\n",
       "      <td>0.548</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.388</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M2D</th>\n",
       "      <th>m2d_vit_base-80x608p16x16-220930-mr7</th>\n",
       "      <td>0.832</td>\n",
       "      <td>0.713</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.361</td>\n",
       "      <td>0.868</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            wacc   uar  r_Present  r_Unknown  \\\n",
       "model ptconf                                                                   \n",
       "AST   pretrained_models                    0.654 0.672      0.744      0.769   \n",
       "BYOLA pretrained_weights                   0.556 0.556      0.590      0.573   \n",
       "Cnn14 external                             0.582 0.548      0.750      0.506   \n",
       "M2D   m2d_vit_base-80x608p16x16-220930-mr7 0.832 0.713      0.911      0.361   \n",
       "\n",
       "                                            r_Absent  count  \n",
       "model ptconf                                                 \n",
       "AST   pretrained_models                        0.505     15  \n",
       "BYOLA pretrained_weights                       0.507     15  \n",
       "Cnn14 external                                 0.388     15  \n",
       "M2D   m2d_vit_base-80x608p16x16-220930-mr7     0.868     15  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_model_name(repr, ptconf):\n",
    "    return repr\n",
    "\n",
    "def read_scores(scorefiles=['scores/circor-scores-rulebased.csv', ]):\n",
    "    df = pd.concat([pd.read_csv(f) for f in scorefiles])\n",
    "\n",
    "    for i, r in df.iterrows():\n",
    "        s = r['weight_file']\n",
    "        org_weightspath = str(s)\n",
    "        ptconf = org_weightspath.split('/')[-2]\n",
    "        if 'bs128a2nr' in s:\n",
    "            #print('NO lo', s)\n",
    "            s = s.replace('bs128a2nr', 'bs128a2lo1.0nr')\n",
    "        #print(s)\n",
    "        m = re.search(r'\\/(m2d.+base).+\\/checkpoint-(\\d+)\\.pth', s)\n",
    "        m = re.search(r\"'seed': (\\d)\", s)\n",
    "        seed = m.group(1) if m is not None else None\n",
    "        model = get_model_name(r['representation'], ptconf)\n",
    "        df.loc[i, 'model'] = model\n",
    "        df.loc[i, 'seed'] = seed\n",
    "        df.loc[i, 'ptconf'] = ptconf\n",
    "        df.loc[i, 'task'] = r['task']\n",
    "\n",
    "    df = df.sort_values('ptconf')\n",
    "    #### EXCLUDE ####\n",
    "    df = df[df.task.isin(['circor1', 'circor4', 'circor5'])]\n",
    "\n",
    "    scoredf = df[['ptconf', 'model', 'wacc', 'uar', 'r_Present', 'r_Unknown', 'r_Absent', ]].groupby(['model', 'ptconf']).mean()\n",
    "    scoredf['count'] = df[['ptconf', 'model', 'uar']].groupby(['model', 'ptconf']).count()\n",
    "    print(len(scoredf), len(df))\n",
    "    return df, scoredf\n",
    "\n",
    "rawscoredf, scoredf = read_scores()\n",
    "print(scoredf.to_latex())\n",
    "scoredf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The ablation results on Table IV (a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 60\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "    &                                      &  wacc &   uar &  r\\_Present &  r\\_Unknown &  r\\_Absent &  count \\\\\n",
      "model & ptconf &       &       &            &            &           &        \\\\\n",
      "\\midrule\n",
      "AST & pretrained\\_models & 0.673 & 0.705 &      0.579 &      0.769 &     0.766 &     15 \\\\\n",
      "BYOLA & pretrained\\_weights & 0.569 & 0.598 &      0.409 &      0.627 &     0.759 &     15 \\\\\n",
      "Cnn14 & external & 0.611 & 0.604 &      0.544 &      0.553 &     0.715 &     15 \\\\\n",
      "M2D & m2d\\_vit\\_base-80x608p16x16-220930-mr7 & 0.796 & 0.683 &      0.794 &      0.314 &     0.940 &     15 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wacc</th>\n",
       "      <th>uar</th>\n",
       "      <th>r_Present</th>\n",
       "      <th>r_Unknown</th>\n",
       "      <th>r_Absent</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>ptconf</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AST</th>\n",
       "      <th>pretrained_models</th>\n",
       "      <td>0.673</td>\n",
       "      <td>0.705</td>\n",
       "      <td>0.579</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.766</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BYOLA</th>\n",
       "      <th>pretrained_weights</th>\n",
       "      <td>0.569</td>\n",
       "      <td>0.598</td>\n",
       "      <td>0.409</td>\n",
       "      <td>0.627</td>\n",
       "      <td>0.759</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cnn14</th>\n",
       "      <th>external</th>\n",
       "      <td>0.611</td>\n",
       "      <td>0.604</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.715</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M2D</th>\n",
       "      <th>m2d_vit_base-80x608p16x16-220930-mr7</th>\n",
       "      <td>0.796</td>\n",
       "      <td>0.683</td>\n",
       "      <td>0.794</td>\n",
       "      <td>0.314</td>\n",
       "      <td>0.940</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            wacc   uar  r_Present  r_Unknown  \\\n",
       "model ptconf                                                                   \n",
       "AST   pretrained_models                    0.673 0.705      0.579      0.769   \n",
       "BYOLA pretrained_weights                   0.569 0.598      0.409      0.627   \n",
       "Cnn14 external                             0.611 0.604      0.544      0.553   \n",
       "M2D   m2d_vit_base-80x608p16x16-220930-mr7 0.796 0.683      0.794      0.314   \n",
       "\n",
       "                                            r_Absent  count  \n",
       "model ptconf                                                 \n",
       "AST   pretrained_models                        0.766     15  \n",
       "BYOLA pretrained_weights                       0.759     15  \n",
       "Cnn14 external                                 0.715     15  \n",
       "M2D   m2d_vit_base-80x608p16x16-220930-mr7     0.940     15  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rawscoredf, scoredf = read_scores(scorefiles=['scores/circor-scores-normal.csv'])\n",
    "print(scoredf.to_latex())\n",
    "scoredf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The ensemble results on Table III"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task</th>\n",
       "      <th>wacc</th>\n",
       "      <th>uar</th>\n",
       "      <th>r_Present</th>\n",
       "      <th>r_Unknown</th>\n",
       "      <th>r_Absent</th>\n",
       "      <th>combinations</th>\n",
       "      <th>models</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.788</td>\n",
       "      <td>0.978</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.856</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.871</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.839</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_cnn...</td>\n",
       "      <td>Cnn14-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>circor4</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.471</td>\n",
       "      <td>0.902</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor4_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>circor4</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.956</td>\n",
       "      <td>0.471</td>\n",
       "      <td>0.874</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor4_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.869</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.862</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_cnn...</td>\n",
       "      <td>Cnn14-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.828</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_cnn...</td>\n",
       "      <td>Cnn14-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.862</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.845</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_cnn...</td>\n",
       "      <td>Cnn14-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.862</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>circor1</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.647</td>\n",
       "      <td>0.845</td>\n",
       "      <td>-lab-physionet2022_dl-evar-logs-circor1_ar_ast...</td>\n",
       "      <td>AST-M2D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        task  wacc   uar  r_Present  r_Unknown  r_Absent  \\\n",
       "81   circor1 0.880 0.788      0.978      0.529     0.856   \n",
       "118  circor1 0.871 0.826      0.933      0.706     0.839   \n",
       "218  circor4 0.869 0.769      0.933      0.471     0.902   \n",
       "350  circor4 0.869 0.767      0.956      0.471     0.874   \n",
       "35   circor1 0.869 0.826      0.911      0.706     0.862   \n",
       "87   circor1 0.867 0.822      0.933      0.706     0.828   \n",
       "31   circor1 0.867 0.795      0.933      0.588     0.862   \n",
       "79   circor1 0.867 0.808      0.933      0.647     0.845   \n",
       "176  circor1 0.867 0.795      0.933      0.588     0.862   \n",
       "187  circor1 0.867 0.808      0.933      0.647     0.845   \n",
       "\n",
       "                                          combinations     models  \n",
       "81   -lab-physionet2022_dl-evar-logs-circor1_ar_ast...    AST-M2D  \n",
       "118  -lab-physionet2022_dl-evar-logs-circor1_ar_cnn...  Cnn14-M2D  \n",
       "218  -lab-physionet2022_dl-evar-logs-circor4_ar_ast...    AST-M2D  \n",
       "350  -lab-physionet2022_dl-evar-logs-circor4_ar_ast...    AST-M2D  \n",
       "35   -lab-physionet2022_dl-evar-logs-circor1_ar_cnn...  Cnn14-M2D  \n",
       "87   -lab-physionet2022_dl-evar-logs-circor1_ar_cnn...  Cnn14-M2D  \n",
       "31   -lab-physionet2022_dl-evar-logs-circor1_ar_ast...    AST-M2D  \n",
       "79   -lab-physionet2022_dl-evar-logs-circor1_ar_cnn...  Cnn14-M2D  \n",
       "176  -lab-physionet2022_dl-evar-logs-circor1_ar_ast...    AST-M2D  \n",
       "187  -lab-physionet2022_dl-evar-logs-circor1_ar_ast...    AST-M2D  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('scores/ensemble-circor-results.csv')\n",
    "\n",
    "comb = df.combinations.values[0]\n",
    "models = comb.split(',')\n",
    "def model_name(model):\n",
    "    if 'ar_byola.AR_BYOLA' in model: return 'BYOL-A'\n",
    "    if 'ar_m2d.AR_M2D' in model: return 'M2D'  # TODO 0.6 ? 0.7 ?\n",
    "    if 'ar_ast.AR_AST' in model: return 'AST'\n",
    "    if 'ar_cnn14.AR_Cnn14' in model: return 'Cnn14'\n",
    "    assert False\n",
    "df['models'] = df.combinations.apply(lambda t: '-'.join(sorted([model_name(m) for m in t.split(',')])))\n",
    "df.sort_values(['wacc'], ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrrr}\n",
      "\\toprule\n",
      "{} &  wacc &   uar &  r\\_Present &  r\\_Unknown &  r\\_Absent \\\\\n",
      "models        &       &       &            &            &           \\\\\n",
      "\\midrule\n",
      "BYOL-A-BYOL-A & 0.575 & 0.581 &      0.619 &      0.616 &     0.508 \\\\\n",
      "Cnn14-Cnn14   & 0.587 & 0.558 &      0.745 &      0.529 &     0.399 \\\\\n",
      "BYOL-A-Cnn14  & 0.587 & 0.580 &      0.652 &      0.582 &     0.505 \\\\\n",
      "AST-BYOL-A    & 0.630 & 0.645 &      0.676 &      0.712 &     0.546 \\\\\n",
      "AST-Cnn14     & 0.648 & 0.652 &      0.734 &      0.700 &     0.523 \\\\\n",
      "AST-AST       & 0.682 & 0.699 &      0.750 &      0.782 &     0.565 \\\\\n",
      "BYOL-A-M2D    & 0.817 & 0.721 &      0.870 &      0.432 &     0.862 \\\\\n",
      "Cnn14-M2D     & 0.829 & 0.719 &      0.898 &      0.391 &     0.868 \\\\\n",
      "AST-M2D       & 0.832 & 0.733 &      0.899 &      0.438 &     0.862 \\\\\n",
      "M2D-M2D       & 0.837 & 0.716 &      0.918 &      0.355 &     0.875 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wacc</th>\n",
       "      <th>uar</th>\n",
       "      <th>r_Present</th>\n",
       "      <th>r_Unknown</th>\n",
       "      <th>r_Absent</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>models</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BYOL-A-BYOL-A</th>\n",
       "      <td>0.575</td>\n",
       "      <td>0.581</td>\n",
       "      <td>0.619</td>\n",
       "      <td>0.616</td>\n",
       "      <td>0.508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cnn14-Cnn14</th>\n",
       "      <td>0.587</td>\n",
       "      <td>0.558</td>\n",
       "      <td>0.745</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BYOL-A-Cnn14</th>\n",
       "      <td>0.587</td>\n",
       "      <td>0.580</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.582</td>\n",
       "      <td>0.505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AST-BYOL-A</th>\n",
       "      <td>0.630</td>\n",
       "      <td>0.645</td>\n",
       "      <td>0.676</td>\n",
       "      <td>0.712</td>\n",
       "      <td>0.546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AST-Cnn14</th>\n",
       "      <td>0.648</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.734</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AST-AST</th>\n",
       "      <td>0.682</td>\n",
       "      <td>0.699</td>\n",
       "      <td>0.750</td>\n",
       "      <td>0.782</td>\n",
       "      <td>0.565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BYOL-A-M2D</th>\n",
       "      <td>0.817</td>\n",
       "      <td>0.721</td>\n",
       "      <td>0.870</td>\n",
       "      <td>0.432</td>\n",
       "      <td>0.862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cnn14-M2D</th>\n",
       "      <td>0.829</td>\n",
       "      <td>0.719</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.391</td>\n",
       "      <td>0.868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AST-M2D</th>\n",
       "      <td>0.832</td>\n",
       "      <td>0.733</td>\n",
       "      <td>0.899</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M2D-M2D</th>\n",
       "      <td>0.837</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.355</td>\n",
       "      <td>0.875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               wacc   uar  r_Present  r_Unknown  r_Absent\n",
       "models                                                   \n",
       "BYOL-A-BYOL-A 0.575 0.581      0.619      0.616     0.508\n",
       "Cnn14-Cnn14   0.587 0.558      0.745      0.529     0.399\n",
       "BYOL-A-Cnn14  0.587 0.580      0.652      0.582     0.505\n",
       "AST-BYOL-A    0.630 0.645      0.676      0.712     0.546\n",
       "AST-Cnn14     0.648 0.652      0.734      0.700     0.523\n",
       "AST-AST       0.682 0.699      0.750      0.782     0.565\n",
       "BYOL-A-M2D    0.817 0.721      0.870      0.432     0.862\n",
       "Cnn14-M2D     0.829 0.719      0.898      0.391     0.868\n",
       "AST-M2D       0.832 0.733      0.899      0.438     0.862\n",
       "M2D-M2D       0.837 0.716      0.918      0.355     0.875"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmpdf = df.groupby('models').mean().sort_values('wacc')\n",
    "print(tmpdf.to_latex())\n",
    "tmpdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Ablation results on Table IV (b) and (c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 75\n",
      "\\begin{tabular}{llrrrrrr}\n",
      "\\toprule\n",
      "    &                                      &  wacc &   uar &  r\\_Present &  r\\_Unknown &  r\\_Absent &  count \\\\\n",
      "model & ptconf &       &       &            &            &           &        \\\\\n",
      "\\midrule\n",
      "AST & pretrained\\_models & 0.670 & 0.617 &      0.772 &      0.490 &     0.590 &     15 \\\\\n",
      "BYOLA & pretrained\\_weights & 0.536 & 0.524 &      0.630 &      0.522 &     0.420 &     15 \\\\\n",
      "Cnn14 & external & 0.374 & 0.374 &      0.550 &      0.447 &     0.126 &     15 \\\\\n",
      "M2D & m2d\\_vit\\_base-80x208p16x16-random & 0.595 & 0.536 &      0.547 &      0.325 &     0.737 &     15 \\\\\n",
      "    & m2d\\_vit\\_base-80x608p16x16-220930-mr7 & 0.832 & 0.713 &      0.911 &      0.361 &     0.868 &     15 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wacc</th>\n",
       "      <th>uar</th>\n",
       "      <th>r_Present</th>\n",
       "      <th>r_Unknown</th>\n",
       "      <th>r_Absent</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>ptconf</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AST</th>\n",
       "      <th>pretrained_models</th>\n",
       "      <td>0.670</td>\n",
       "      <td>0.617</td>\n",
       "      <td>0.772</td>\n",
       "      <td>0.490</td>\n",
       "      <td>0.590</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BYOLA</th>\n",
       "      <th>pretrained_weights</th>\n",
       "      <td>0.536</td>\n",
       "      <td>0.524</td>\n",
       "      <td>0.630</td>\n",
       "      <td>0.522</td>\n",
       "      <td>0.420</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cnn14</th>\n",
       "      <th>external</th>\n",
       "      <td>0.374</td>\n",
       "      <td>0.374</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.447</td>\n",
       "      <td>0.126</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">M2D</th>\n",
       "      <th>m2d_vit_base-80x208p16x16-random</th>\n",
       "      <td>0.595</td>\n",
       "      <td>0.536</td>\n",
       "      <td>0.547</td>\n",
       "      <td>0.325</td>\n",
       "      <td>0.737</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>m2d_vit_base-80x608p16x16-220930-mr7</th>\n",
       "      <td>0.832</td>\n",
       "      <td>0.713</td>\n",
       "      <td>0.911</td>\n",
       "      <td>0.361</td>\n",
       "      <td>0.868</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            wacc   uar  r_Present  r_Unknown  \\\n",
       "model ptconf                                                                   \n",
       "AST   pretrained_models                    0.670 0.617      0.772      0.490   \n",
       "BYOLA pretrained_weights                   0.536 0.524      0.630      0.522   \n",
       "Cnn14 external                             0.374 0.374      0.550      0.447   \n",
       "M2D   m2d_vit_base-80x208p16x16-random     0.595 0.536      0.547      0.325   \n",
       "      m2d_vit_base-80x608p16x16-220930-mr7 0.832 0.713      0.911      0.361   \n",
       "\n",
       "                                            r_Absent  count  \n",
       "model ptconf                                                 \n",
       "AST   pretrained_models                        0.590     15  \n",
       "BYOLA pretrained_weights                       0.420     15  \n",
       "Cnn14 external                                 0.126     15  \n",
       "M2D   m2d_vit_base-80x208p16x16-random         0.737     15  \n",
       "      m2d_vit_base-80x608p16x16-220930-mr7     0.868     15  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rawscoredf, scoredf = read_scores(scorefiles=['scores/circor-scores-ablations.csv'])\n",
    "print(scoredf.to_latex())\n",
    "scoredf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
